package util;

import java.sql.DriverManager;
import java.util.List;

import javax.management.JMException;

import com.mysql.jdbc.Connection;
import com.mysql.jdbc.PreparedStatement;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

public class Zhihu implements PageProcessor {
	private Site site = Site.me().setCycleRetryTimes(5).setRetryTimes(5).setSleepTime(100).setTimeOut(3 * 60 * 1000)
			.setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0")
			.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
			.addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3").setCharset("UTF-8");

	private static final int voteNum = 500;
	public static final String url = "jdbc:mysql://127.0.0.1/SearchEngine?useUnicode=true&characterEncoding=UTF-8";
	public static final String name = "com.mysql.jdbc.Driver";
	public static final String user = "root";
	public static final String password = "33505832";

	public Connection conn = null;
	public PreparedStatement pst = null;

	@Override
	public void process(Page page) {
		List<String> relativeUrl = page.getHtml().xpath("//li[@class='item clearfix']/div/a/@href").all();
		page.addTargetRequests(relativeUrl);
		relativeUrl = page.getHtml()
				.xpath("//div[@id='zh-question-related-questions']//a[@class='question_link']/@href").all();
		page.addTargetRequests(relativeUrl);
		List<String> answers = page.getHtml().xpath("//div[@id='zh-question-answer-wrap']/div").all();
		boolean exist = false;
		for (String answer : answers) {
			String vote = new Html(answer).xpath("//div[@class='zm-votebar']//span[@class='count']/text()").toString();
			if (Integer.valueOf(vote) >= voteNum) {
				page.putField("title", page.getHtml().xpath("title/text()"));
				page.putField("relatedUrl",
						page.getHtml()
								.xpath("//div[@class='zh-question-related-questions clearfix']//a[@class='question_link']/@href")
								.all());
				page.putField("relatedTitle",
						page.getHtml()
								.xpath("//div[@class='zh-question-related-questions clearfix']//a[@class='question_link']/text()")
								.all());
				page.putField("vote", vote);
				page.putField("content", new Html(answer).xpath("//div[@class='zm-editable-content']/text()"));
				page.putField("userid", new Html(answer).xpath("//a[@class='author-link']/@href"));
				exist = true;
				// 构建并且存入Mysql
				cn.zju.pojo.Pages p = new cn.zju.pojo.Pages();
				p.setTitle(page.getHtml().xpath("title/text()").toString());
				p.setUrl(page.getUrl().toString());
				p.setContent(new Html(answer).xpath("//div[@class='zm-editable-content']/text()").toString());
				p.setVote(Integer.valueOf(vote));
				p.setUserid(new Html(answer).xpath("//a[@class='author-link']/@href").toString());
				p.setRelateUrl(page.getHtml()
						.xpath("//div[@class='zh-question-related-questions clearfix']//a[@class='question_link']/@href")
						.all().toString());
				p.setRelateTitle(page.getHtml()
						.xpath("//div[@class='zh-question-related-questions clearfix']//a[@class='question_link']/text()")
						.all().toString());
				try {
					Class.forName(name);
					// 指定连接类型
					conn = (Connection) DriverManager.getConnection(url, user, password);// 获取连接
					String sql = "INSERT INTO pages (title,url,content,vote,userid,relate_url,relate_title)VALUES(\""
							+ p.getTitle() + "\",\" " + p.getUrl() + "\",\"" + p.getContent() + "\",\" " + p.getVote()
							+ "\",\" " + p.getUserid() + "\",\" " + p.getRelateUrl() + "\",\" " + p.getRelateTitle()
							+ "\")";
					pst = (PreparedStatement) conn.prepareStatement(sql);// 准备执行语句
					pst.execute();
					this.conn.close();
					this.pst.close();
				} catch (Exception e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
		}
		if (!exist) {
			page.setSkip(true);
		}
	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) throws JMException {
		Spider spider = Spider.create(new Zhihu()).addUrl("https://www.zhihu.com/search?type=content&q=人际")
				.addPipeline(new ConsolePipeline()).thread(5);
		spider.run();
	}
}
